package com.iiitb.wtp;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.StringTokenizer;
import java.util.concurrent.CopyOnWriteArrayList;

import org.apache.commons.collections.HashBag;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.params.ConnRoutePNames;
import org.apache.http.impl.client.DefaultHttpClient;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

public class WordFreqCounter {
	static int countWords = 0;

	
	public static boolean parse(String content) {
		Document doc = Jsoup.parse(content);
		String token = "";
		List<String> parseWords = new CopyOnWriteArrayList<String>();
		Elements ele = doc.getElementsByTag("body");
		// Elements ele = doc.getElementsByClass("infobox vcard");
		Document doc1 = Jsoup.parse(ele.toString());

		String body = doc1.body().text();
		StringTokenizer strtok = new StringTokenizer(body);

		while (strtok.hasMoreTokens()) {
			token = strtok.nextToken().replaceAll("[^\\p{Alpha}]+", "");
			if (!token.equals("")) {
				token = token.toLowerCase();
				countWords++;
				parseWords.add(token);
				
			}

		}
		boolean humanCategory = removeStopWords(parseWords);
		return humanCategory;
		// Bag<String> wordFreq =

	}

	private static boolean removeStopWords(List<String> parseWords) {
		Stopwords sw = new Stopwords();
		// Frequency freq = new Frequency();
		HashBag freqTable = new HashBag();
		for (String string : parseWords) {
			if (sw.is(string)) {
				parseWords.remove(string);
			}

		}
		System.out.println("Removed the stopwords successfully");

		for (String string : parseWords) {
			freqTable.add(string);
		}
		ArrayList<String> HumankeyWords = new ArrayList<String>();
		// ArrayList<String> FandFkeyWords = new ArrayList<String>();
		Stem pjForKey = new Stem(
				"stemrules.txt", "/p");
		HumankeyWords = pjForKey
				.ReadKeywords("C:/Users/V/workspace/Classification/src/HumanCategory.txt");
		// FandFkeyWords = pjForKey
		// .ReadKeywords("C:/Users/V/workspace/Classification/src/FandFCategory.txt");
		Map<String, Integer> HumanmapKey = new HashMap<String, Integer>();
		// Map<String, Integer> FandFmapKey = new HashMap<String, Integer>();

		HashBag HumanlemmatizedKeyWords = new HashBag();
		String lem;
		for (String str : HumankeyWords) {
			lem = pjForKey.lemmatize(str);
			HumanlemmatizedKeyWords.add(str);
		}

		int count = 0;
		Iterator iter1 = HumanlemmatizedKeyWords.uniqueSet().iterator();
		while (iter1.hasNext()) {
			String value = (String) iter1.next();
			// pj.lemmatize(value);
			count = HumanlemmatizedKeyWords.getCount(value);
			HumanmapKey.put(value, count);

		}


		  Map<String, Integer> map = new HashMap<String, Integer>();
		  HashBag lemmatizedHTMLWords = new HashBag(); 
		
		   Iterator iter =  freqTable.iterator();
		   while (iter.hasNext()) { String value =
		  (String) iter.next();
		    count = freqTable.getCount(value);
		  lemmatizedHTMLWords.add(value);
		  
		  } // int count = 0;
		   iter1 =  lemmatizedHTMLWords.uniqueSet().iterator();
		   while (iter1.hasNext()) {
		  String value = (String) iter1.next();
		   //pj.lemmatize(value);
		   count =  lemmatizedHTMLWords.getCount(value);
		  
		  if (count > 1) { map.put(value, count); } } Iterator iterator =
		  map.keySet().iterator(); while (iterator.hasNext()) { Object key =
		  iterator.next(); }
	
		  int flag =  0;
		  int noOfHumanMatches = 0; // int noOfFandFMatches = 0;
		  Iterator  iterator1 = map.keySet().iterator(); 
		  while (iterator1.hasNext()) {
			  Object key = iterator1.next(); 
			  if (HumanmapKey.containsKey(key)) {
				  	noOfHumanMatches++;
				  	flag = 1; 
			  } 
		  }
		  if (flag == 1 && noOfHumanMatches > 2){// &&
		 System.out.println("HUMAN category"); return true; 
		  }
		
		 else{
		System.out.println("Neither category");
		return false;
		 }

	}

	private static Map sortByComparator(Map unsortMap) {

		List list = new LinkedList(unsortMap.entrySet());

		// sort list based on comparator
		Collections.sort(list, new Comparator() {
			public int compare(Object o1, Object o2) {
				return ((Comparable) ((Map.Entry) (o2)).getValue())
						.compareTo(((Map.Entry) (o1)).getValue());
			}
		});

		// put sorted list into map again
		// LinkedHashMap make sure order in which keys were inserted
		Map sortedMap = new LinkedHashMap();
		for (Iterator it = list.iterator(); it.hasNext();) {
			Map.Entry entry = (Map.Entry) it.next();
			sortedMap.put(entry.getKey(), entry.getValue());
		}
		return sortedMap;
	}

	public static void printMap(Map<String, Integer> map) {
		for (Map.Entry entry : map.entrySet()) {
			System.out.println("Key : " + entry.getKey() + " Value : "
					+ entry.getValue());
		}
	}

}